{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "3287220f-be37-49fe-8c72-e94851c5899f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import sys\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n",
    "logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))\n",
    "\n",
    "os.environ[\"OPENAI_API_KEY\"] = os.getenv('OPENAI_API_KEY')\n",
    "\n",
    "import openai"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "930beb2d-578f-4ad0-8675-d2d74513cdb8",
   "metadata": {},
   "source": [
    "# Configuring ElasticsearchStore"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88814a18-36dd-444c-9777-10590528f9ce",
   "metadata": {},
   "source": [
    "# Load documents, build VectorStoreIndex with Elasticsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "05b88d74-c3e4-43b2-ad2e-41b87f7a5881",
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index import VectorStoreIndex, SimpleDirectoryReader\n",
    "from llama_index.vector_stores import ElasticsearchStore"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8dfeb63-03ec-4272-9a4b-d7c84d667ba6",
   "metadata": {},
   "source": [
    "## Download data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "5133a919-e65c-4e72-bb51-724d72e050c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-01-17 14:36:42--  https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/paul_graham/paul_graham_essay.txt\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 75042 (73K) [text/plain]\n",
      "Saving to: ‘data/paul_graham/paul_graham_essay.txt’\n",
      "\n",
      "data/paul_graham/pa 100%[===================>]  73.28K   323KB/s    in 0.2s    \n",
      "\n",
      "2024-01-17 14:36:43 (323 KB/s) - ‘data/paul_graham/paul_graham_essay.txt’ saved [75042/75042]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!mkdir -p 'data/paul_graham/'\n",
    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/paul_graham/paul_graham_essay.txt' -O 'data/paul_graham/paul_graham_essay.txt'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dcfce6c6-7261-4d7c-8bea-6811d0823473",
   "metadata": {},
   "source": [
    "## load documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "98137d01-4e02-463a-a9c8-6ed6c63c6803",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:elastic_transport.transport:GET http://localhost:9200/ [status:200 duration:0.010s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.010s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.010s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.010s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.010s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.010s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.010s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.010s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.010s]\n",
      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "INFO:elastic_transport.transport:HEAD http://localhost:9200/paul_graham [status:404 duration:0.013s]\n",
      "HEAD http://localhost:9200/paul_graham [status:404 duration:0.013s]\n",
      "HEAD http://localhost:9200/paul_graham [status:404 duration:0.013s]\n",
      "HEAD http://localhost:9200/paul_graham [status:404 duration:0.013s]\n",
      "HEAD http://localhost:9200/paul_graham [status:404 duration:0.013s]\n",
      "HEAD http://localhost:9200/paul_graham [status:404 duration:0.013s]\n",
      "HEAD http://localhost:9200/paul_graham [status:404 duration:0.013s]\n",
      "HEAD http://localhost:9200/paul_graham [status:404 duration:0.013s]\n",
      "HEAD http://localhost:9200/paul_graham [status:404 duration:0.013s]\n",
      "INFO:elastic_transport.transport:PUT http://localhost:9200/paul_graham [status:200 duration:0.101s]\n",
      "PUT http://localhost:9200/paul_graham [status:200 duration:0.101s]\n",
      "PUT http://localhost:9200/paul_graham [status:200 duration:0.101s]\n",
      "PUT http://localhost:9200/paul_graham [status:200 duration:0.101s]\n",
      "PUT http://localhost:9200/paul_graham [status:200 duration:0.101s]\n",
      "PUT http://localhost:9200/paul_graham [status:200 duration:0.101s]\n",
      "PUT http://localhost:9200/paul_graham [status:200 duration:0.101s]\n",
      "PUT http://localhost:9200/paul_graham [status:200 duration:0.101s]\n",
      "PUT http://localhost:9200/paul_graham [status:200 duration:0.101s]\n",
      "INFO:elastic_transport.transport:PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.056s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.056s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.056s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.056s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.056s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.056s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.056s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.056s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.056s]\n",
      "INFO:elastic_transport.transport:PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.032s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.032s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.032s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.032s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.032s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.032s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.032s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.032s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.032s]\n"
     ]
    }
   ],
   "source": [
    "documents = SimpleDirectoryReader(\"./data/paul_graham/\").load_data()\n",
    "\n",
    "from llama_index.storage.storage_context import StorageContext\n",
    "\n",
    "vector_store = ElasticsearchStore(\n",
    "    es_url=\"http://localhost:9200\",\n",
    "    # Or with Elastic Cloud\n",
    "    # es_cloud_id=\"my_cloud_id\",\n",
    "    # es_user=\"elastic\",\n",
    "    # es_password=\"my_password\",\n",
    "    index_name=\"paul_graham\",\n",
    ")\n",
    "\n",
    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
    "index = VectorStoreIndex.from_documents(\n",
    "    documents, storage_context=storage_context\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fd22f19-6f35-4e86-ae6a-3e5bb3cce450",
   "metadata": {},
   "source": [
    "# Basic query example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "01a073c7-b652-445f-9993-8b4bf92e83da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "INFO:elastic_transport.transport:POST http://localhost:9200/paul_graham/_search [status:200 duration:0.020s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.020s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.020s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.020s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.020s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.020s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.020s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.020s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.020s]\n",
      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "He invested his own money to fund Y Combinator.\n"
     ]
    }
   ],
   "source": [
    "# set Logging to DEBUG for more detailed outputs\n",
    "query_engine = index.as_query_engine()\n",
    "response = query_engine.query(\"what were his investments in Y Combinator?\")\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "331b3136-7739-4ba7-ab69-63e4568a9246",
   "metadata": {},
   "source": [
    "# Metadata Filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "70a34f55-b8be-4250-a623-1fabc799561f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:elastic_transport.transport:GET http://localhost:9200/ [status:200 duration:0.009s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.009s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.009s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.009s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.009s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.009s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.009s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.009s]\n",
      "GET http://localhost:9200/ [status:200 duration:0.009s]\n",
      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "INFO:elastic_transport.transport:HEAD http://localhost:9200/movies_metadata_example [status:404 duration:0.012s]\n",
      "HEAD http://localhost:9200/movies_metadata_example [status:404 duration:0.012s]\n",
      "HEAD http://localhost:9200/movies_metadata_example [status:404 duration:0.012s]\n",
      "HEAD http://localhost:9200/movies_metadata_example [status:404 duration:0.012s]\n",
      "HEAD http://localhost:9200/movies_metadata_example [status:404 duration:0.012s]\n",
      "HEAD http://localhost:9200/movies_metadata_example [status:404 duration:0.012s]\n",
      "HEAD http://localhost:9200/movies_metadata_example [status:404 duration:0.012s]\n",
      "HEAD http://localhost:9200/movies_metadata_example [status:404 duration:0.012s]\n",
      "HEAD http://localhost:9200/movies_metadata_example [status:404 duration:0.012s]\n",
      "INFO:elastic_transport.transport:PUT http://localhost:9200/movies_metadata_example [status:200 duration:0.095s]\n",
      "PUT http://localhost:9200/movies_metadata_example [status:200 duration:0.095s]\n",
      "PUT http://localhost:9200/movies_metadata_example [status:200 duration:0.095s]\n",
      "PUT http://localhost:9200/movies_metadata_example [status:200 duration:0.095s]\n",
      "PUT http://localhost:9200/movies_metadata_example [status:200 duration:0.095s]\n",
      "PUT http://localhost:9200/movies_metadata_example [status:200 duration:0.095s]\n",
      "PUT http://localhost:9200/movies_metadata_example [status:200 duration:0.095s]\n",
      "PUT http://localhost:9200/movies_metadata_example [status:200 duration:0.095s]\n",
      "PUT http://localhost:9200/movies_metadata_example [status:200 duration:0.095s]\n",
      "INFO:elastic_transport.transport:PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.051s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.051s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.051s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.051s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.051s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.051s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.051s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.051s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.051s]\n",
      "INFO:elastic_transport.transport:PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.015s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.015s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.015s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.015s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.015s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.015s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.015s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.015s]\n",
      "PUT http://localhost:9200/_bulk?refresh=true [status:200 duration:0.015s]\n",
      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "INFO:elastic_transport.transport:POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.016s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.016s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.016s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.016s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.016s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.016s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.016s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.016s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.016s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[NodeWithScore(node=TextNode(id_='1c023db8-edb9-48b7-8b06-ac2de1fa0ace', embedding=None, metadata={'director': 'Mark shuttle', 'theme': 'Mafia'}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, hash='cd0d432dd8907ae25697c3a94c8294b70185be23e73ec3ad93f316a3362a9cdc', text='Beautiful weather', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=1.0),\n",
       " NodeWithScore(node=TextNode(id_='44597058-43d8-4162-8ffe-34a58163eef8', embedding=None, metadata={'director': 'Francis Ford Coppola', 'theme': 'Mafia'}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, hash='81cf4b9e847ba42e83fc401e31af8e17d629f0d5cf9c0c320ec7ac69dd0257e1', text='The Godfather', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.9427951049198355)]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from llama_index.schema import TextNode\n",
    "\n",
    "nodes = [\n",
    "    TextNode(\n",
    "        text=\"The Shawshank Redemption\",\n",
    "        metadata={\n",
    "            \"author\": \"Stephen King\",\n",
    "            \"theme\": \"Friendship\",\n",
    "        },\n",
    "    ),\n",
    "    TextNode(\n",
    "        text=\"The Godfather\",\n",
    "        metadata={\n",
    "            \"director\": \"Francis Ford Coppola\",\n",
    "            \"theme\": \"Mafia\",\n",
    "        },\n",
    "    ),\n",
    "    TextNode(\n",
    "        text=\"Beautiful weather\",\n",
    "        metadata={\n",
    "            \"director\": \"Mark shuttle\",\n",
    "            \"theme\": \"Mafia\",\n",
    "        },\n",
    "    ),    \n",
    "    TextNode(\n",
    "        text=\"Inception\",\n",
    "        metadata={\n",
    "            \"director\": \"Christopher Nolan\",\n",
    "        },\n",
    "    ),\n",
    "]\n",
    "\n",
    "# initialize the vector store\n",
    "vector_store_metadata_example = ElasticsearchStore(\n",
    "    index_name=\"movies_metadata_example\",\n",
    "    es_url=\"http://localhost:9200\",\n",
    ")\n",
    "storage_context = StorageContext.from_defaults(\n",
    "    vector_store=vector_store_metadata_example\n",
    ")\n",
    "index1 = VectorStoreIndex(nodes, storage_context=storage_context)\n",
    "\n",
    "\n",
    "# Metadata filter\n",
    "from llama_index.vector_stores.types import ExactMatchFilter, MetadataFilters\n",
    "\n",
    "filters = MetadataFilters(\n",
    "    filters=[ExactMatchFilter(key=\"theme\", value=\"Mafia\")]\n",
    ")\n",
    "\n",
    "retriever = index1.as_retriever(filters=filters)\n",
    "\n",
    "retriever.retrieve(\"weather is so beautiful\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "4c55a85c-f91e-4604-b16f-9d2d1e3b00d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "INFO:elastic_transport.transport:POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.015s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.015s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.015s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.015s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.015s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.015s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.015s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.015s]\n",
      "POST http://localhost:9200/movies_metadata_example/_search [status:200 duration:0.015s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[NodeWithScore(node=TextNode(id_='44597058-43d8-4162-8ffe-34a58163eef8', embedding=None, metadata={'director': 'Francis Ford Coppola', 'theme': 'Mafia'}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, hash='81cf4b9e847ba42e83fc401e31af8e17d629f0d5cf9c0c320ec7ac69dd0257e1', text='The Godfather', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=1.0),\n",
       " NodeWithScore(node=TextNode(id_='1c023db8-edb9-48b7-8b06-ac2de1fa0ace', embedding=None, metadata={'director': 'Mark shuttle', 'theme': 'Mafia'}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={}, hash='cd0d432dd8907ae25697c3a94c8294b70185be23e73ec3ad93f316a3362a9cdc', text='Beautiful weather', start_char_idx=None, end_char_idx=None, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=0.9803472140656279)]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retriever.retrieve(\"The godfather is a nice person\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47668d77-efe0-431b-8e4c-24df32b57220",
   "metadata": {},
   "source": [
    "# Custom Filters and overriding Query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "82ee9a9a-8ebe-469b-bde9-74e729e8d026",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
      "custom query {'knn': {'filter': [{'match': {'content': 'growing up'}}], 'field': 'embedding', 'query_vector': [0.0026147642638534307, -0.03282194584608078, 0.01616211235523224, -0.02955910563468933, -0.006743430159986019, 0.016300367191433907, -0.03702492266893387, 0.0023157859686762094, -0.002994967857375741, -0.0035497196950018406, 0.03273899108171463, 0.030305687338113785, 0.011717185378074646, 0.004994493443518877, 0.0031228545121848583, 0.016189763322472572, 0.018125345930457115, -0.008537299931049347, 0.020835161209106445, -0.011627319268882275, -0.027678826823830605, -0.004669592250138521, -0.004745632875710726, -0.015360227786004543, -0.006518764421343803, 0.012837057933211327, 0.023309940472245216, -0.00935300998389721, -0.0026596973184496164, 0.003012249944731593, 0.03249013051390648, 0.0014888430014252663, -0.0023002321831882, 0.02380766160786152, -0.029282594099640846, 0.003567001549527049, 0.00482167350128293, 0.01943877711892128, 0.01823594979941845, -0.009408311918377876, 0.03777150437235832, 0.026254791766405106, -0.016784263774752617, -0.005046339705586433, -0.020655427128076553, -0.012837057933211327, -0.0020427305717021227, -0.019245218485593796, -0.0026406871620565653, 0.01632801815867424, -0.0002473483909852803, 0.027457617223262787, -0.011972959153354168, 0.013521424494683743, -0.005993392318487167, 0.0029828704427927732, -0.005201877560466528, -0.004569356795400381, -0.012401551939547062, 0.003788210917264223, 0.03459161892533302, 0.020766032859683037, 4.371802060632035e-05, 0.010210196487605572, -0.02015770599246025, -0.003147049341350794, 0.003525524865835905, -0.027277885004878044, 0.01617593690752983, 0.016535403206944466, 0.04681343957781792, -0.00013760779984295368, 0.01689486764371395, -0.0020254484843462706, 0.014268006198108196, 0.0006191271240822971, -0.014558344148099422, -0.014240355230867863, -0.026019755750894547, 0.021305231377482414, 0.00219999672845006, -0.034536316990852356, -0.027996815741062164, -0.00955348089337349, 0.0004808712110389024, -0.0019113875459879637, -0.0002814803156070411, 0.020199183374643326, -0.021858254447579384, 0.0035635451786220074, 0.04407597333192825, 0.03879459947347641, 0.021692346781492233, 0.022964300587773323, -0.03290489688515663, -0.007099439390003681, -0.012346250005066395, -0.009525829926133156, -0.013763372786343098, -0.004686874337494373, -0.01610681042075157, -0.0023175140377134085, -0.015899425372481346, -0.025425255298614502, -0.018927229568362236, 0.003074465086683631, 0.012657325714826584, 0.004776740446686745, 0.022687789052724838, -0.027900036424398422, -0.004783653188496828, 0.006190406624227762, -0.018208298832178116, -0.0263515692204237, -0.021236103028059006, -0.023793837055563927, 0.02507961541414261, 0.01952173002064228, -0.007949712686240673, -0.014115924946963787, 0.03215831518173218, -0.0026787077076733112, 0.016825739294290543, -0.0071893054991960526, 0.017973264679312706, 0.0077907186932861805, -0.027775606140494347, -0.008993544615805149, 0.010383016429841518, -0.03202006220817566, 0.013051354326307774, 0.011136510409414768, 0.02254953421652317, -0.01501458790153265, -0.027858559042215347, 0.025217872112989426, 0.0034650377929210663, 0.0026579692494124174, -0.033651478588581085, -0.04310818389058113, -0.002832517260685563, 0.005602819379419088, -0.008336829021573067, 0.030969316139817238, -0.010929127223789692, 0.016784263774752617, 0.003556632436811924, 0.03672076016664505, 0.0016037682071328163, -0.002281222026795149, 0.004683417733758688, -0.017074599862098694, 0.009007370099425316, 0.012926924042403698, -0.013853238895535469, 0.0012010979698970914, -0.0008178699063137174, 0.009622609242796898, -0.014876332134008408, -0.018913403153419495, -0.01170335989445448, 0.005945002660155296, 0.002039274200797081, 0.015042238868772984, 0.0019062028732150793, 0.04626041650772095, 0.05519174411892891, 0.0007470137788914144, -0.003656867891550064, -0.009975161403417587, -0.010548923164606094, 0.02366940677165985, -0.037550296634435654, 0.007984276860952377, 0.004216804169118404, -0.0028774503152817488, 0.0037812981754541397, -0.01199369691312313, -0.0361400842666626, 0.008516562171280384, 0.015982380136847496, -0.00595191540196538, 0.023448197171092033, 0.026876943185925484, 0.015788821503520012, -0.02725023403763771, -0.002106674015522003, -0.0334579236805439, -0.011530539952218533, 0.005606275983154774, 0.0005547517212107778, 0.005841310601681471, -0.008039578795433044, 0.013638942502439022, -0.6357558369636536, -0.019798241555690765, -0.006715779192745686, -0.02238362655043602, 0.01227020937949419, -0.00868938211351633, 0.008979719132184982, 0.016770437359809875, -0.015553786419332027, 0.015042238868772984, 0.01405370980501175, 0.01533257681876421, -0.028300978243350983, -0.006328662857413292, 0.014738076366484165, -0.02644834853708744, 0.029669711366295815, -0.042416904121637344, -0.005983022972941399, 0.025051964446902275, -0.02301960252225399, 0.004334321711212397, 0.0030312601011246443, 0.006173125002533197, 0.015512309037148952, 0.017475543543696404, -0.0005560478311963379, -0.003295674454420805, -0.02159556746482849, 0.019231392070651054, -0.015000762417912483, 0.039237018674612045, 0.009684824384748936, 0.021014893427491188, 0.039707086980342865, 0.009622609242796898, -0.01159275509417057, 0.014406261965632439, 0.011060469783842564, 0.04205743595957756, -0.025549685582518578, -0.008993544615805149, 0.027208756655454636, 0.010942952707409859, -0.003798580262809992, -0.00671232258901, 0.034923434257507324, 0.0028290608897805214, -0.024084173142910004, -0.0017463444964960217, 0.018982531502842903, -0.002391826594248414, -0.02348967269062996, 0.00883455015718937, 0.005398891866207123, -0.015457007102668285, 0.004272106569260359, -0.007299910299479961, -0.0017454804619774222, -0.000628200126811862, -0.0022535708267241716, 0.02126375399529934, -0.02277074195444584, -0.026973720639944077, 0.005900069605559111, 0.007514206692576408, -0.030637502670288086, 0.013431558385491371, 0.016438623890280724, -0.01721285656094551, 0.015125192701816559, 0.020489521324634552, -0.007451991550624371, 0.004552074708044529, -0.005599362775683403, 0.017807357013225555, 0.02867426909506321, -0.008599515073001385, -0.0036706936080008745, 0.020904287695884705, -0.022646311670541763, -0.012906186282634735, -0.017503194510936737, -0.01617593690752983, 0.008309178054332733, -0.006653564050793648, -0.008350654505193233, 0.002498975023627281, 0.0038607954047620296, -0.024167127907276154, 0.010894563049077988, 0.004386167507618666, 0.007852933369576931, -0.013625117018818855, 0.010500533506274223, 0.02939319983124733, 0.022162416949868202, 0.009608783759176731, 0.014641297049820423, -0.021443486213684082, -0.01863689161837101, 0.010154894553124905, 0.014668948017060757, 0.001663391012698412, -0.01047979574650526, -0.006124735344201326, 0.008730858564376831, 0.021526439115405083, -0.0010472883004695177, -0.018913403153419495, -0.009159451350569725, -0.005011775530874729, -0.011883092112839222, -0.004057810176163912, -0.013728808611631393, -0.03655485436320305, 0.0332920141518116, 0.0066189998760819435, 0.009795429185032845, -0.0164801012724638, 0.012615848332643509, -0.02778943069279194, 0.026821639388799667, -0.009636434726417065, -0.006238796282559633, 0.01353524997830391, 0.025383779779076576, -0.01639714650809765, -0.020752206444740295, 0.013058267533779144, 0.0037398214917629957, -0.022328324615955353, 0.017033124342560768, -0.015346402302384377, -0.008869114331901073, -0.006594805512577295, 0.002452313667163253, -0.020351264625787735, -0.012740278616547585, -0.02054482325911522, -0.030609851703047752, -0.005454194266349077, -0.007113264873623848, 0.024153301492333412, -0.033983293920755386, -0.0022898628376424313, 0.026185663416981697, 0.012767930515110493, -0.017351113259792328, 0.016687484458088875, 0.027927687391638756, 0.009539655409753323, -0.0197429396212101, 0.019784415140748024, 0.020295962691307068, -0.008516562171280384, -0.011675708927214146, -0.03495108708739281, -0.01920374110341072, -0.02183060348033905, 0.011599668301641941, 0.011890005320310593, -0.01672896184027195, -0.0006208553095348179, 0.001421443186700344, -0.02516257017850876, 0.012588197365403175, 0.011426848359405994, -0.004794022534042597, -0.006135104689747095, 0.0015450094360858202, -0.019176090136170387, -0.0044483826495707035, 0.006155842915177345, -0.013376256451010704, 0.036361295729875565, 0.009408311918377876, 0.0063563138246536255, -0.01414357591420412, 0.01482103019952774, -0.003936836030334234, 0.003629216691479087, -0.004545161966234446, 0.010583487339317799, 0.029282594099640846, -0.01354216318577528, 0.002701174234971404, 0.02303342893719673, -0.00480093527585268, -0.0026061232201755047, -0.01905165985226631, 0.004196065943688154, -0.005087816156446934, 0.013867064379155636, 0.0016452448908239603, -0.006086715031415224, 0.02629626728594303, 0.021139323711395264, 0.012546720914542675, 0.0016011758707463741, 0.012048999778926373, 0.03249013051390648, 0.01689486764371395, -0.008668643422424793, -0.015221972018480301, -0.032849594950675964, -0.015968553721904755, -0.00864099245518446, 0.008744684047996998, 0.008226224221289158, 0.0056546651758253574, 0.0018526287749409676, -0.009276969358325005, -0.005170769989490509, 0.01689486764371395, -0.0015432812506332994, -0.004655766766518354, 0.02914433926343918, -0.012131952680647373, 0.01609298400580883, -0.00023481895914301276, -0.0124222906306386, 0.013466122560203075, -0.03459161892533302, 0.011219464242458344, 0.004254824481904507, -0.0018059674184769392, 0.007417427841573954, 0.00307619315572083, -0.0028256045188754797, -0.003967943601310253, 0.02062777616083622, 0.009124888107180595, -0.004047440830618143, 0.022687789052724838, 0.0032386439852416515, 0.027595872059464455, -0.02507961541414261, 0.038822248578071594, -0.0031090290285646915, 0.004158045630902052, 0.018899578601121902, 0.019093137234449387, 0.007486555725336075, 0.022093288600444794, -0.000645914173219353, 0.04111729934811592, 0.002201724797487259, -0.0014240355230867863, 0.015567611902952194, 1.1908367014257237e-05, 0.0012736822245642543, 0.012290947139263153, -0.02574324421584606, -0.002130868611857295, -0.02478927932679653, 0.016673658043146133, 0.010410667397081852, 0.018194472417235374, -0.000585427216719836, 0.013549075461924076, -0.012678063474595547, 0.008965893648564816, 0.0031505057122558355, 0.01322417426854372, -0.00669158436357975, 0.0028100507333874702, -0.007894410751760006, 0.016742786392569542, -0.022286847233772278, 0.03403859585523605, -0.01801474019885063, 0.021609393879771233, 0.011489063501358032, -0.0016564782708883286, 0.005920807830989361, 0.002882634988054633, 0.004738720133900642, -0.018111519515514374, 0.0022967758122831583, -0.008005015552043915, -0.04797478765249252, 0.014627471566200256, 0.010984429158270359, 0.005606275983154774, 0.008157096803188324, 0.022715440019965172, 0.01665983349084854, -0.0029379373881965876, 0.016300367191433907, -0.008516562171280384, -0.024180952459573746, -0.013085918501019478, 0.011979871429502964, 0.01567821577191353, -0.022162416949868202, 0.033651478588581085, -0.033817388117313385, -0.0170607753098011, -0.017641449347138405, -0.00012172997958259657, 0.008060317486524582, -0.0013782383175566792, 0.009304620325565338, 0.04335704445838928, 0.00900045782327652, -0.00743125332519412, -0.0027270971331745386, -0.01672896184027195, -0.028536012396216393, 0.019134612753987312, -0.01043140608817339, -0.03179885074496269, 0.014558344148099422, 0.020613951608538628, 0.009290794841945171, -0.009836905635893345, -0.012166516855359077, 0.011357720009982586, 0.012284034863114357, -0.020586300641298294, -0.007403602357953787, -0.03619538992643356, 0.012837057933211327, 0.1080607920885086, 0.029973873868584633, -0.025134919211268425, 0.0054438249208033085, 0.0018197930185124278, -0.0010775317205116153, -0.02151261456310749, -0.036914318799972534, 0.020669253543019295, -0.0008265108917839825, 0.004251368343830109, 0.000974703929387033, 0.01378411054611206, 0.010141068138182163, 0.010756307281553745, -0.024872232228517532, -0.002924111671745777, -0.0185677632689476, -0.0028774503152817488, -0.03832452744245529, -0.014198878780007362, -0.0049184528179466724, 0.01679808832705021, 0.031494688242673874, 0.022978127002716064, 0.018457159399986267, 0.049412649124860764, 0.01493163499981165, 0.03481283038854599, -0.01975676417350769, 0.005848223343491554, 0.0082815270870924, 0.007237695157527924, 0.014226529747247696, -0.014599820598959923, -0.0006446180050261319, -0.02263248711824417, 0.032047711312770844, 0.004503685049712658, 0.004002507776021957, 0.025356126949191093, 0.013756459578871727, 0.0133831687271595, -0.018028566613793373, -0.00038214787491597235, -0.0025750156491994858, 0.003527252934873104, 0.011717185378074646, 0.007168567273765802, -0.012698802165687084, 0.013611290603876114, -0.0072722588665783405, -0.033181410282850266, -0.014005320146679878, 0.0016858575399965048, 0.012698802165687084, -0.004489859566092491, -0.015913251787424088, -0.013127395883202553, -0.015913251787424088, -0.010666441172361374, -0.03439806029200554, 0.023697057738900185, -0.009014283306896687, 0.0198258925229311, -0.018899578601121902, -0.02231449820101261, -0.030969316139817238, -0.027582047507166862, -0.016521576792001724, 0.004434557165950537, 0.005578624550253153, -0.0434952974319458, -0.006124735344201326, 0.024471290409564972, 0.03074810653924942, 0.014834855683147907, 0.010998254641890526, -0.020918114110827446, -0.0023382524959743023, 0.011095033958554268, -0.017586147412657738, -0.007313735783100128, -0.01143376063555479, -0.003506514709442854, -0.005319394636899233, -0.000818301981780678, -0.00699229072779417, 0.004731807392090559, 0.004427644424140453, -0.005554429721087217, 0.004133850801736116, -0.008495823480188847, -0.0371355265378952, 0.0027443792205303907, -0.01354216318577528, -0.005478389095515013, 0.0018318903166800737, 0.011226377449929714, -0.018692193552851677, 0.0012702258536592126, 0.003527252934873104, -0.0033699870109558105, 0.006577523425221443, -0.021001067012548447, 0.015221972018480301, 0.004745632875710726, 0.02317168563604355, -0.03990064561367035, 0.009477440267801285, 0.02303342893719673, -0.016438623890280724, -0.0007362125325016677, 0.03058220073580742, -0.007728503551334143, 0.018941054120659828, -0.004517510533332825, -0.0006610358832404017, -0.018457159399986267, -0.007500381208956242, 0.01450304128229618, -0.01594090275466442, 0.03257308527827263, 0.009346096776425838, -0.0281903725117445, -0.003992138430476189, -0.0012244286481291056, -0.03495108708739281, -0.014171226881444454, 0.028010640293359756, -0.037246134132146835, 0.02588150091469288, -0.027844732627272606, -0.02183060348033905, -0.026171837002038956, -0.03196476027369499, -0.039624135941267014, -0.005499127320945263, -0.0008874299237504601, -0.004728350788354874, 0.0013393538538366556, -0.029973873868584633, 0.010224021971225739, -0.033817388117313385, -0.017793530598282814, -0.030886363238096237, -0.01607915759086609, 0.001029142178595066, 0.004811304621398449, 0.03354087471961975, -0.003978312946856022, 0.013362430036067963, -0.02643452398478985, -0.023462021723389626, -0.022438928484916687, -0.022051813080906868, 0.019535554572939873, -0.020420392975211143, 0.03832452744245529, 0.01553996093571186, 0.032711341977119446, -0.0008571864455007017, 0.03481283038854599, 0.020613951608538628, 0.013804849237203598, -0.006079802289605141, -0.0062111453153193, 0.00011362904479028657, -0.0004050465067848563, 0.02725023403763771, -0.010659527964890003, 0.0031712441705167294, 0.01429565716534853, 0.014627471566200256, -0.006349401082843542, 0.018263600766658783, -0.00647037522867322, 0.007873672060668468, -0.01482103019952774, -0.02597827836871147, -0.023379068821668625, 0.005958828143775463, -0.011012080125510693, -0.01824977621436119, -0.01952173002064228, 0.011164162307977676, 0.003012249944731593, 0.02501048892736435, 0.024250080808997154, -0.009304620325565338, -0.0013039257610216737, 0.0002907693851739168, 0.019065484404563904, 0.0027582047041505575, 0.021872078999876976, -0.012726453132927418, -0.002934481017291546, -0.021775299683213234, -0.029337896034121513, 0.015111367218196392, -0.0033976382110267878, 0.011088121682405472, 0.0124637670814991, -0.012242557480931282, 0.0041027432307600975, 0.0022086375392973423, -0.024415988475084305, -0.008972805924713612, 0.019300520420074463, -0.025287000462412834, -0.016148285940289497, -0.027416139841079712, -0.0165630541741848, -0.010922214016318321, -0.0003080513852182776, -0.014765727333724499, 0.00043658612412400544, -0.0030917469412088394, -0.011599668301641941, -0.013479948043823242, -0.026572778820991516, 0.0023244270123541355, 0.03622303903102875, -0.01793178729712963, 0.006494569592177868, 8.009119483176619e-05, 0.008136358112096786, -0.016120634973049164, -0.0038296878337860107, 0.006936988793313503, 0.01139228418469429, 0.010465970262885094, 0.03320906311273575, -0.02159556746482849, -0.0199088454246521, 0.008585689589381218, 0.006007217802107334, -0.016618356108665466, -0.0037709290627390146, -0.011986784636974335, -0.019549380987882614, -0.002602666849270463, 0.0003045949852094054, -0.029448501765727997, -0.02079368382692337, -0.005544060841202736, 0.002863624831661582, 0.0059622847475111485, -0.00011049668682971969, -0.01446156483143568, -0.01138537097722292, 1.0119411854248028e-05, 0.012339336797595024, -0.00408200453966856, 0.012353162281215191, -0.005734162405133247, 0.004147676285356283, -0.03658250346779823, 0.005388522986322641, 0.03354087471961975, -0.004818217363208532, 0.007673201151192188, -0.014807204715907574, 0.027996815741062164, -0.009249318391084671, 0.004455295391380787, -0.029503803700208664, -0.014323309063911438, -0.008855288848280907, 0.02454041875898838, 0.013320953585207462, 0.0028359736315906048, -0.006840209476649761, -0.021084021776914597, 0.006978465244174004, -0.01398458145558834, -0.004545161966234446, -0.022425103932619095, -0.019812067970633507, 0.006266447715461254, 0.020406566560268402, 0.015913251787424088, -0.009816166944801807, 0.027305535972118378, -0.004164958372712135, 0.0013609562302008271, 0.014544517733156681, -0.01533257681876421, 0.00388153363019228, -0.028618967160582542, -0.01493163499981165, -0.03213066607713699, -0.0007189305615611374, -0.0006411616341210902, -0.013244912959635258, -0.0033907254692167044, 0.004762914963066578, 0.004438013769686222, 0.00297595770098269, 0.009733214043080807, -0.004683417733758688, 0.0020634690299630165, -0.02978031523525715, 0.02014388144016266, -0.011502888984978199, 9.569898975314572e-05, 0.023282289505004883, 0.0015692041488364339, -0.015622913837432861, 0.004216804169118404, 0.005716880317777395, -0.0028567120898514986, 0.011226377449929714, 0.001423171372152865, 0.0008278070599772036, 0.02109784632921219, 0.014779552817344666, -0.02931024506688118, -0.008952068164944649, 0.01195222046226263, -0.02317168563604355, -0.0033838124945759773, 0.02126375399529934, -0.021789126098155975, 0.012678063474595547, 0.007327561266720295, 0.014337134547531605, 0.010742481797933578, -0.02859131433069706, -0.014337134547531605, -0.0012131952680647373, -0.014392436482012272, 0.030084477737545967, 0.005976110231131315, -0.022203894332051277, -0.015498483553528786, -0.016466274857521057, -0.0021343249827623367, 0.00817092228680849, 0.0016487013781443238, 0.01918991468846798, 0.01350068673491478, 0.012394639663398266, 0.02389061450958252, -0.01681191474199295, -0.008509648963809013, -0.0263515692204237, -0.025522034615278244, -0.03132878243923187, -0.00792897492647171, -0.014406261965632439, 0.02738848887383938, 0.0023123295977711678, -0.023061079904437065, -0.04319113492965698, -0.0017852289602160454, -0.027374664321541786, -0.022660138085484505, 0.005267548840492964, 0.0023244270123541355, 0.03074810653924942, -0.019936498254537582, -0.028563663363456726, 0.027457617223262787, 0.00042384068365208805, 0.03420450538396835, 0.0018474441021680832, -0.01179322600364685, 0.026683384552598, -0.007265346124768257, 0.010147981345653534, -0.021844428032636642, -0.017821181565523148, -0.007631724234670401, 0.007313735783100128, 0.0062733604572713375, 0.008931329473853111, 0.005257179494947195, 0.005734162405133247, 0.007445078808814287, -0.002632046351209283, 0.018609240651130676, -0.013341692276299, 0.03887755423784256, 0.00513966241851449, -0.03243482857942581, -0.019881194457411766, 0.002644143532961607, -0.008419782854616642, -0.002994967857375741, 0.012933837249875069, -0.020530996844172478, -0.003534165909513831, -0.02348967269062996, -0.002080750884488225, -0.005108554847538471, -0.010624963790178299, 0.03323671221733093, 0.005578624550253153, -0.024180952459573746, 0.015567611902952194, 0.020130055025219917, 0.009076498448848724, -0.017337286844849586, -0.0009012554655782878, 0.02111167274415493, -0.005436912178993225, 0.0027789431624114513, 0.014240355230867863, -0.015443181619048119, 0.0066708461381495, 0.001672032056376338, 0.003698344575241208, -0.013576727360486984, -0.003677606349810958, -0.005194964352995157, -0.0022483861539512873, 0.025604987516999245, 0.01594090275466442, 0.004095830488950014, -0.003174700541421771, 0.015871774405241013, -0.005917351692914963, 0.010410667397081852, -0.021567916497588158, -0.009677911177277565, -0.005893156863749027, -0.02000562474131584, -0.010694092139601707, 0.018816623836755753, -0.017904136329889297, -0.033651478588581085, -0.007617898751050234, 0.026959896087646484, -0.009823080152273178, -0.009007370099425316, 0.2216518223285675, 0.0029206553008407354, 0.007576421834528446, 0.020116230472922325, 0.00010747234045993537, 0.01497311145067215, 0.027056675404310226, 0.025300825014710426, -0.007832195609807968, 0.029089035466313362, -0.006981921847909689, 0.015705866739153862, 0.0012261568335816264, -0.008654817938804626, -0.007244607899338007, -0.0035428067203611135, -0.03442571312189102, 0.006833296734839678, -0.03503403812646866, 0.011281679384410381, 0.01791796088218689, -0.008647904731333256, 0.0041856965981423855, -0.02014388144016266, -0.004697243217378855, 0.013825587928295135, -0.003878077259287238, 0.003247284796088934, 0.015982380136847496, 0.030084477737545967, -0.009657173417508602, 0.00011033467308152467, -0.003316412912681699, -0.029448501765727997, 0.016051506623625755, -0.03514464199542999, 0.009982074610888958, -0.0012512156972661614, 0.01912078820168972, -0.0032196338288486004, 0.0001572660548845306, 0.008288439363241196, -0.02812124602496624, -0.0033094999380409718, 0.00016860736650414765, 0.004897714592516422, -0.02111167274415493, -0.006857491564005613, 0.01350068673491478, 0.004538249224424362, -0.00500140618532896, 0.013334779068827629, 0.042886972427368164, 0.016632182523608208, -0.018042391166090965, 0.01374263409525156, 0.037384387105703354, 0.007292997557669878, -0.014696599915623665, 0.020420392975211143, -0.008475084789097309, 0.01871984638273716, 0.005464563611894846, 0.001487978850491345, 0.010341539978981018, -0.019646160304546356, -0.004814760759472847, 0.016300367191433907, 0.026144186034798622, 0.00109999836422503, 0.004285932052880526, -0.0022034530993551016, -0.013659680262207985, 0.010044289752840996, -0.04498846083879471, -0.027291709557175636, 0.04244455322623253, 0.03757794573903084, 0.021084021776914597, 0.016466274857521057, 0.025287000462412834, 0.014724250882863998, 0.01027932483702898, 0.016742786392569542, -0.01482103019952774, -0.035974178463220596, 0.021554091945290565, -0.0164801012724638, -0.009684824384748936, -0.026019755750894547, 0.0056235576048493385, 0.0008684197091497481, -0.008744684047996998, -0.029033733531832695, 0.007417427841573954, -0.014751901850104332, -0.029033733531832695, 0.02661425620317459, 0.004299757536500692, 0.017157554626464844, -0.02613036148250103, 0.017572320997714996, 0.019646160304546356, 0.013590552844107151, -0.016908694058656693, 0.0005366055993363261, -0.009670998901128769, 0.00964334700256586, 0.008150183595716953, -0.00980234146118164, -0.005575168412178755, -0.005295200273394585, 0.02397356927394867, 0.012373900972306728, 0.012214906513690948, 0.017005473375320435, -0.0049184528179466724, -0.02507961541414261, 0.03567001596093178, -0.00258020032197237, -0.014641297049820423, -0.04073018208146095, -0.01823594979941845, 0.009076498448848724, -0.0066155437380075455, -0.0021550634410232306, -0.011979871429502964, -0.02366940677165985, -0.012560546398162842, -0.026019755750894547, 0.03865634277462959, 0.015705866739153862, 0.004807848017662764, -0.019397299736738205, 0.00434123445302248, -0.03464692085981369, 0.009546568617224693, -0.009988986887037754, 0.008454347029328346, -0.009221666492521763, -0.023074906319379807, 0.00776998046785593, -0.0015692041488364339, -0.010922214016318321, 0.005661577917635441, -0.03943057730793953, 0.01800091564655304, 0.014489215798676014, 0.018941054120659828, -0.021305231377482414, -0.009456701576709747, 0.007783805951476097, -0.016314193606376648, -0.018609240651130676, -0.012691889889538288, -0.014350960031151772, -0.024319209158420563, -0.05013158172369003, 0.006242252886295319, 0.0020271767862141132, -0.03121817670762539, 0.015595262870192528, 0.02286752127110958, -0.024747801944613457, -0.024651022627949715, -0.004766371101140976, -0.17763115465641022, 0.017143728211522102, 0.013555988669395447, 0.0004679097328335047, 0.04001124948263168, 0.004372342023998499, 0.021139323711395264, -0.01107429526746273, -0.018028566613793373, 0.016590705141425133, 0.016300367191433907, 0.0026942614931613207, -0.01824977621436119, -0.03276664391160011, -0.002763389376923442, 0.020503345876932144, 0.00026549448375590146, 0.009740126319229603, 0.03121817670762539, 0.007134003099054098, 0.01751701906323433, -0.027665000408887863, -0.008101793937385082, 0.01649392582476139, 0.005730706267058849, -0.0023140576668083668, 0.003497873665764928, 0.010465970262885094, -0.0082815270870924, -0.030637502670288086, -0.000632520648650825, 0.0018439877312630415, 0.03757794573903084, -0.009670998901128769, 0.021844428032636642, -0.0003549287503119558, -0.009187103249132633, -0.023932091891765594, 0.0019511360442265868, 0.0018474441021680832, 0.012505244463682175, 0.03567001596093178, 0.0020565560553222895, 0.013314041309058666, -0.023213161155581474, 0.0034926889929920435, 0.024664849042892456, -0.018982531502842903, 0.004064722917973995, 0.010846173390746117, 0.025784721598029137, -0.052288372069597244, 0.016383321955800056, -0.00478019705042243, 0.013155046850442886, -0.009505091235041618, 0.017862658947706223, 0.002243201481178403, -0.008302265778183937, -0.004469121340662241, 0.001546737621538341, -0.012325511313974857, 0.011765575036406517, -0.03464692085981369, -0.010811609216034412, -0.025383779779076576, -0.007514206692576408, 0.001939038629643619, -0.037163179367780685, 0.023309940472245216, -0.0019148439168930054, -0.003964487463235855, 0.0202406607568264, 0.008309178054332733, 0.003124582814052701, 0.006446180399507284, -0.003746734233573079, 0.049910370260477066, 0.005284830927848816, -0.013576727360486984, -0.012090476229786873, 0.043218787759542465, 0.002652784576639533, 0.005315938498824835, -0.004372342023998499, 0.024581894278526306, 0.018305078148841858, 0.00034067113301716745, -0.009698649868369102, -0.022397451102733612, 0.0092838816344738, 0.0009228579583577812, -0.0012849156046286225, -0.005215703044086695, 0.006342488341033459, 0.01800091564655304, -0.0027703021187335253, 0.000878788938280195, 0.003102116286754608, -0.014599820598959923, -0.011675708927214146, -0.01350068673491478, -0.006473831366747618, -0.013092831708490849, 0.03506169095635414, 0.0035324376076459885, -0.015304925851523876, 0.03193710744380951, 0.020987242460250854, -0.0026631539221853018, -0.027512919157743454, 0.016438623890280724, 0.02022683434188366, 0.026890767738223076, 0.0006048694485798478, 0.022411277517676353, -0.006594805512577295, -0.023531150072813034, 0.009518916718661785, -0.030084477737545967, 0.04905318468809128, 0.007984276860952377, -0.017420239746570587, -0.013334779068827629, 0.005367784295231104, -0.00541271734982729, -0.10098209232091904, 1.8618637113831937e-05, 0.019563207402825356, 0.0015562426997348666, 0.0006817742832936347, 0.03334731608629227, 0.016314193606376648, 0.021374357864260674, -0.0371355265378952, 0.04805774241685867, -0.001958048902451992, -0.035476457327604294, -0.013452297076582909, 0.020060928538441658, -0.012636587023735046, 0.01665983349084854, 0.0044138189405202866, -0.00677108159288764, -0.025909151881933212, 0.021070195361971855, -0.009055759757757187, -0.025674115866422653, -5.10898680659011e-05, -0.010009725578129292, 0.004154589027166367, 0.0027478355914354324, -0.029531454667448997, 0.011060469783842564, 0.005105098243802786, 0.006767624989151955, 0.0022933194413781166, -0.021001067012548447, 0.0170607753098011, -0.021305231377482414, 0.0038607954047620296, 0.009698649868369102, -0.026807814836502075, -0.024402162060141563, 0.02842540852725506, -0.006553328596055508, 0.0071409158408641815, 0.020185358822345734, 0.02062777616083622, -0.03705257549881935, -0.022991951555013657, 0.00852347444742918, -0.0038296878337860107, 0.003345792181789875, 0.01457216963171959, -0.003846969688311219, -0.013804849237203598, 0.010058115236461163, -0.010147981345653534, -0.01425418071448803, 0.018830450251698494, -0.015913251787424088, 0.01251215673983097, 0.023793837055563927, -0.020199183374643326, -0.005208790302276611, 0.00035471274168230593, -0.014917808584868908, -0.0164801012724638, -0.005637383554130793, 0.014876332134008408, -0.028563663363456726, -0.02485840581357479, -0.018664542585611343, -0.00396103085950017, -0.035808272659778595, -0.0028601684607565403, 0.04338469356298447, -0.02223154529929161, 0.01634184457361698, -0.019715288653969765, -0.022355975583195686, -0.00388153363019228, -0.017033124342560768, 0.017724404111504555, -0.010507446713745594, -0.00305545493029058, -0.024706324562430382, 0.00023719524324405938, -0.015457007102668285, 0.03113522380590439, 0.006860947702080011, 0.0029604039154946804, -0.0063217501156032085, -0.00467996159568429, -0.005004862789064646, -0.02318551018834114, -0.0032300029415637255, 0.012194167822599411, -0.02054482325911522, -0.012588197365403175, 0.013244912959635258, 0.004330865107476711, -0.023600278422236443, -0.005333220586180687, 0.025701766833662987, -0.014765727333724499, 0.029890920966863632, -0.052067164331674576, 0.03752264380455017, -0.01927286945283413, -0.00987146981060505, -0.004095830488950014, -0.011876179836690426, 0.013286389410495758, -0.009512004442512989, -0.0169778224080801, 0.009684824384748936, -0.034066248685121536, 0.012892360799014568, -0.004472577478736639, -0.007106352131813765, -0.0029413937591016293, -0.018374206498265266, 0.02539760433137417, -0.04418657720088959, 0.004154589027166367, 0.029669711366295815, 0.01943877711892128, -0.003836600575596094, 0.027360837906599045, -0.0017161010764539242, 0.002101489342749119, 0.0006351129268296063, -0.03282194584608078, 0.012201081030070782, -0.011820876970887184, -0.013562901876866817, 0.009007370099425316, -0.007037224248051643, -0.009781603701412678, -0.005716880317777395, 0.003954118117690086, -0.008267701603472233, -0.004071635659784079, 0.01736493781208992, 0.00956039410084486, 0.010078852996230125, -0.01960468292236328, -0.0124222906306386, 0.0025594618637114763, -0.01032080128788948, -0.01672896184027195, 0.0026631539221853018, -0.000517595442943275, 0.0040543535724282265, 0.02238362655043602, 0.013120482675731182, -0.008094881661236286, 0.0008524339064024389, -0.013632029294967651, -0.021554091945290565, 0.0027893122751265764, -0.004372342023998499, 0.010175632312893867, -0.015111367218196392, -0.002842886373400688, -0.01027932483702898, 0.020724555477499962, -0.006594805512577295, -0.002592297736555338, -0.02827332727611065, 0.025383779779076576, -0.008039578795433044, 0.0005983887240290642, -0.00541963055729866, 0.03514464199542999, -0.026088884100317955, -0.02301960252225399, -0.015263448469340801, 0.0019960692152380943, 0.02651747688651085, -0.013950018212199211, -0.018125345930457115, 0.02549438364803791, -0.02478927932679653, -0.013528337702155113, 0.018844276666641235, -0.005253723356872797, -0.02382148802280426, 0.010687178932130337, -0.010224021971225739, 0.028646618127822876, -0.004956473130732775, -0.0019442233024165034, -0.003577370895072818, 0.014447739347815514, 0.023213161155581474, 0.013286389410495758, 0.009249318391084671, -0.01298222690820694, -0.042665764689445496, 0.0021360532846301794, 0.009982074610888958, 0.005094728898257017, -0.001033462700434029, -0.016950169578194618, 0.015968553721904755, 0.0072722588665783405, 0.007472730241715908, 0.005053252447396517, -0.02141583524644375, -0.02606123313307762, 0.012920011766254902, 0.003750190604478121, -0.019024008885025978, -0.015111367218196392, 0.029448501765727997, 0.031245827674865723, 0.011890005320310593, -0.016784263774752617, 0.008620253764092922, -0.0057030548341572285, 0.003912641201168299, 0.012933837249875069, -0.03215831518173218, -0.0002311465359525755, 0.03702492266893387, 0.03265603631734848, 0.01251215673983097, 0.02556351199746132, -0.017254333943128586, 0.0336238294839859, 0.0021965401247143745, -0.00523989787325263, -0.01163423154503107, 0.014668948017060757, -0.007203130982816219, -0.016452448442578316, -0.0024661391507834196, 0.011613493785262108, -0.010293150320649147, -0.013459209352731705, -0.03276664391160011, 0.008765422739088535, 0.0115443654358387, 0.006259534973651171, 0.06962566077709198, -0.005741075146943331, -0.018830450251698494, 0.00904193427413702, 0.007223869673907757, 0.010175632312893867, -0.008703207597136497, 0.003022619057446718, -0.02859131433069706, -0.01920374110341072, 0.020821334794163704, -0.008606428280472755, 0.01569204218685627, -0.01303061656653881, -0.0115443654358387, 0.00797045137733221, 0.0217476487159729, 0.04070252925157547, -0.03854573890566826, 0.006964639760553837, 0.014012233354151249, 0.02972501330077648, -0.0006809101905673742, -0.001487978850491345, -0.003850426059216261, -0.00679181981831789, 0.020890463143587112, -0.00194076681509614, -0.020904287695884705, -0.041227903217077255, 0.003195438999682665, -0.005934633314609528, -0.042112741619348526, -0.013300214894115925, 0.0002255299041280523, -0.01529110036790371, 0.00508090341463685, -0.013929279521107674, 0.043467648327350616, 0.0038642517756670713, -0.021319055929780006, 0.007120177615433931, -0.009131800383329391, -0.013562901876866817, 0.039707086980342865, 0.014890157617628574, -0.013991494663059711, -0.002129140542820096, -0.009657173417508602], 'k': 2, 'num_candidates': 20}}\n",
      "INFO:elastic_transport.transport:POST http://localhost:9200/paul_graham/_search [status:200 duration:0.017s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.017s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.017s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.017s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.017s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.017s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.017s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.017s]\n",
      "POST http://localhost:9200/paul_graham/_search [status:200 duration:0.017s]\n",
      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
      "He invested his own money to fund Y Combinator.\n"
     ]
    }
   ],
   "source": [
    "def custom_query(query, query_str):\n",
    "    print(\"custom query\", query)\n",
    "    return query\n",
    "\n",
    "query_engine = index.as_query_engine(\n",
    "    vector_store_kwargs={\n",
    "        \"es_filter\": [{\"match\": {\"content\": \"growing up\"}}],\n",
    "        \"custom_query\": custom_query,\n",
    "    }\n",
    ")\n",
    "\n",
    "response = query_engine.query(\"what were his investments in Y Combinator?\")\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0418fe22-aebb-45d4-9dcf-ec6944911e00",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b69cc3db-15aa-41ce-897e-1230b87c0cfd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python_3.11",
   "language": "python",
   "name": "python_3.11"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
